# merge the raw http records to be with payload info
# the task to get all http records

require "00_common"

HTTP_RAW_DIR = "./http_raw/"
HTTP_PAYLOAD_DIR = "./http_payload/"

module MergeHttpPayloads
  def self.run
    puts ">> start to get http records..."

    begin
      all_files = MergeHttpPayloads.get_all_files
      puts all_files
      MergeHttpPayloads.handle_all_files all_files

      puts ">> end to get http records."
    rescue Exception => e
      puts ">> EEROR:"
      puts e.to_s
    end
  end


  def self.get_all_files
    Common.get_all_files(HTTP_RAW_DIR)
  end

  def self.merge_value(mapping ,key, value)
    if mapping.has_key? key
      if !(mapping[key].downcase.include? value.downcase)
        mapping[key] = mapping[key] + "-$-" + value
      end
    else
      mapping[key] = value
    end

    mapping
  end

  def self.handle_multi_payload(payload_str, is_client, mapping)
    payload_mapping = Hash.new
    payload_mapping.merge! mapping

    multi_payload = payload_str.split "\\r\\n\\r\\n"
    
    multi_payload.each do |sinple_payload|
      fields = sinple_payload.split "\\r\\n"
      fields.each do |field|
        if field.include? ": "
          key = ""
          value = ""

          pair = field.split ": "

          if !pair[0] or pair[0].length == 0
            key = "NULL"
          else
            key = pair[0].strip.downcase
          end

          if !pair[1] or pair[1].length == 0
            value = "NULL"
          else
            value = pair[1].strip
          end

          # if the key exist, add string to the end of value
          payload_mapping = merge_value(payload_mapping, key, value)

        elsif is_client
          payload_mapping = merge_value(payload_mapping, "c_request", field)
        else
          payload_mapping = merge_value(payload_mapping, "s_response", field)
        end
      end
    end

    payload_mapping
  end

  def self.extract_payload(client_payload, server_payload)
    payload_mapping = Hash.new

    if client_payload and client_payload.length != 0 and client_payload != "NULL"
      payload_mapping = handle_multi_payload(client_payload, true, payload_mapping)
    end

    if server_payload and server_payload.length != 0 and server_payload != "NULL"
      payload_mapping = handle_multi_payload(server_payload, false, payload_mapping)
    end

    payload_mapping
  end

  def self.handle_each_line(line)
    fields1 = line.split "|"

    # from http records
    # new_line = "#{local_time_stamp}|#{session_id}|#{session_start_time}|#{client_ip}|#{server_ip}"
    # new_line = new_line + "|#{l4_proto}|#{client_port}|#{server_port}|#{protocol}|#{client_payload}|#{server_payload}"

    local_time_stamp    = fields1[0]
    session_id          = fields1[1]
    session_start_time  = fields1[2]
    client_ip           = fields1[3]
    server_ip           = fields1[4]
    # l4 = fields1[5]
    client_port         = fields1[6]
    server_port         = fields1[7]
    protocol            = fields1[8]
    client_payload      = fields1[9]
    server_payload      = fields1[10]

    if protocol.downcase != "http"
      return nil
    else
      payload_mapping = extract_payload(client_payload, server_payload)

      payload_mapping["local_time_stamp"]   = local_time_stamp
      payload_mapping["session_id"]         = session_id
      payload_mapping["session_start_time"] = session_start_time
      payload_mapping["server_ip"]          = server_ip
      payload_mapping["server_port"]        = server_port
      payload_mapping["client_ip"]          = client_ip
      payload_mapping["client_port"]        = client_port

      payload_mapping
    end
  end


  def self.handle_each_file(file_name)
    puts "now handling : " + file_name
    file = File.open HTTP_RAW_DIR + file_name, "r"
    output_file_name = Common.alternate_file_name(file_name, "_raw_", "_payload_")
    output_file = File.open output_file_name, "w"

    begin
      counter = 0
      while line = file.readline
        counter = counter + 1
        if counter % 100000 == 0
          puts counter
        end

        if line.start_with? "#"
          next
        end

        payload_mapping = handle_each_line(line)
        if payload_mapping
          local_time_stamp    = payload_mapping["local_time_stamp"]
          session_id          = payload_mapping["session_id"]
          session_start_time  = payload_mapping["session_start_time"]
          server_ip           = payload_mapping["server_ip"]
          server_port         = payload_mapping["server_port"]
          client_ip           = payload_mapping["client_ip"]
          client_port         = payload_mapping["client_port"]

          referer             = Common.check_string(payload_mapping["referer"])
          host                = Common.check_string(payload_mapping["host"])
          content_type        = Common.check_string(payload_mapping["content-type"])
          content_length      = Common.check_string(payload_mapping["content-length"])
          user_agent          = Common.check_string(payload_mapping["user-agent"])
          connection          = Common.check_string(payload_mapping["connection"])

          cookie              = Common.check_string payload_mapping["cookie"]
          # Cache-Control

          c_request           = Common.check_string(payload_mapping["c_request"])
          s_response          = Common.check_string(payload_mapping["s_response"])

          new_line = "#{local_time_stamp}|#{session_id}|#{session_start_time}|#{server_ip}|#{server_port}|#{client_ip}|#{client_port}"
          new_line = new_line + "|#{referer}|#{host}|#{content_type}|#{content_length}|#{user_agent}|#{connection}|#{c_request}|#{s_response}|#{cookie}"

          output_file.puts new_line
        end
      end
    rescue Exception => e
      puts e.to_s
      file.close
    end

    output_file.close
  end

  def self.handle_all_files(all_files)
    all_files.each do |file_name|
      if Common.file_exist?(HTTP_PAYLOAD_DIR, Common.alternate_file_name(file_name, "_raw_", "_payload_"))
        puts "exist file:" + file_name
      else
        handle_each_file(file_name)
        Common.move_file Common.alternate_file_name(file_name, "_raw_", "_payload_"), HTTP_PAYLOAD_DIR
      end
    end
  end

end
